{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###第三周作业\n",
    "在 Rental Listing Inquiries 数据上练习 xgboost 参数调优\n",
    "数据说明：\n",
    "Rental Listing Inquiries 数据集是 Kaggle 平台上的一个分类竞赛任务，需要根据\n",
    "公寓的特征来预测其受欢迎程度（用户感兴趣程度分为高、中、低三类）。其\n",
    "中房屋的特征 x 共有 14 维，响应值 y 为用户对该公寓的感兴趣程度。评价标准\n",
    "为 logloss。\n",
    "数据链接：https://www.kaggle.com/c/two-sigma-connect-rental-listing-inquiries\n",
    "为减轻大家对特征工程的入手难度，以及统一标准，数据请用课程网站提供的\n",
    "特征工程编码后的数据（RentListingInquries_FE_train.csv）或稀疏编码的形式\n",
    "（RentListingInquries_FE_train.bin）。xgboost 既可以单独调用，也可以在\n",
    "sklearn 框架下调用。大家可以随意选择。若采用 xgboost 单独调用使用方式，\n",
    "建议读取稀疏格式文件。\n",
    "关于特征工程的过程，可参看文件：FE_RentListingInqueries.ipynb\n",
    "作业要求：\n",
    "采用 xgboost 模型完成商品分类（需进行参数调优）。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier # scikit-learn嵌入的\n",
    "import xgboost as xgb             #用xgboost，交叉验证做迭代次数\n",
    "\n",
    "import numpy  as np #线性代数等\n",
    "import pandas as pd #读取文件等\n",
    "import scipy  as sp #处理10^10级别的数等\n",
    "import seaborn           as sns       #画图      \n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from matplotlib import pyplot\n",
    "from sklearn.model_selection import GridSearchCV  #网格搜索加交叉验证\n",
    "from sklearn.model_selection import StratifiedKFold #缺省的交叉验证用\n",
    "\n",
    "from sklearn.metrics import log_loss #评价指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取数据\n",
    "trainRental = pd.read_csv(\"RentListingInquries_FE_train.csv\")\n",
    "testRental  = pd.read_csv(\"RentListingInquries_FE_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#trainRental.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#trainRental.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#testRental.head()#观察之后发现trainRental和testRental差异在于　一维　响应值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#testRental.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sns.countplot(trainRental.interest_level)#查看响应值分布，两两之间都相差不到10倍，还可以"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#划分输入特征和响应\n",
    "X_train = trainRental.drop(\"interest_level\",axis = 1)\n",
    "y_train = trainRental[\"interest_level\"]\n",
    "#y_train.head()\n",
    "cols = X_train.columns\n",
    "#print(cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#已经是特征编码后的数据\n",
    "#StratifiedKFold 分层采样交叉切分，确保训练集，测试集中各类别样本的比例与原始数据集中相同。\n",
    "RentalKFold = StratifiedKFold(n_splits = 4,shuffle = True,random_state = 3)\n",
    "#待观察模型是过拟合还是欠拟合"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一轮调整得到的迭代次数为383"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': [3, 5, 7, 9], 'min_child_weight': [1, 3, 5]}\n"
     ]
    }
   ],
   "source": [
    "#依据建议来设置参数\n",
    "max_depth = list(range(3,10,2))\n",
    "min_child_weight = list(range(1,6,2))\n",
    "param_test_2 = dict(max_depth = max_depth,min_child_weight = min_child_weight)\n",
    "print(param_test_2)#构造网格"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbSecond = XGBClassifier(                               #三步曲第一步\n",
    "                        learning_rate = 0.1,\n",
    "                        n_estimators = 383,    #首次变更项\n",
    "                        max_depth = 4,      \n",
    "                        min_child_weight = 1,   #同一考虑\n",
    "                        gamma = 0,\n",
    "                        subsample = 0.3,\n",
    "                        colsample_bytree = 0.8,\n",
    "                        colsample_bylevel = 0.7,#行列采样\n",
    "                        objective = \"multi:softprob\",                                       #因为是多类分问题\n",
    "                        seed = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=4, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=4, min_child_weight=1, missing=None, n_estimators=383,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.3),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [3, 5, 7, 9], 'min_child_weight': [1, 3, 5]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#GSearch_2 = GridSearchCV(xgbSecond,param_grid = param_test_2，scoring = \"neg_log_loss\",n_jobs = -1,cv = RentalKFold )\n",
    "GSearch_2 = GridSearchCV(xgbSecond, param_grid = param_test_2, scoring='neg_log_loss',n_jobs= -1, cv=RentalKFold)\n",
    "\n",
    "GSearch_2.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59462, std: 0.00333, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59520, std: 0.00330, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.59456, std: 0.00369, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.59021, std: 0.00458, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58922, std: 0.00537, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58907, std: 0.00497, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.60277, std: 0.00542, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.59987, std: 0.00408, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.59841, std: 0.00447, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.63461, std: 0.00913, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.62276, std: 0.00682, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.61804, std: 0.00596, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 5},\n",
       " -0.5890653738428216)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GSearch_2.grid_scores_, GSearch_2.best_params_,     GSearch_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 181.37291527,  178.53121579,  182.12101084,  284.01519299,\n",
       "         301.03689659,  298.88649124,  399.55104864,  427.64188415,\n",
       "        4565.73749316,  483.80206692,  490.57809061,  492.83050156]),\n",
       " 'mean_score_time': array([ 0.94802123,  0.90628558,  0.91443187,  2.89581305,  3.17381489,\n",
       "         2.26978648,  7.89085698,  6.57723844,  8.2069478 , 11.7004869 ,\n",
       "        11.25217056, 10.37032503]),\n",
       " 'mean_test_score': array([-0.59462457, -0.59520393, -0.59456281, -0.59021073, -0.58921574,\n",
       "        -0.58906537, -0.60276885, -0.59987498, -0.59840915, -0.63461086,\n",
       "        -0.62275713, -0.61804233]),\n",
       " 'mean_train_score': array([-0.55641783, -0.55761853, -0.55846289, -0.46200644, -0.4711924 ,\n",
       "        -0.47752031, -0.31956259, -0.35179659, -0.37205426, -0.17714011,\n",
       "        -0.23054977, -0.2662311 ]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'rank_test_score': array([ 5,  6,  4,  3,  2,  1,  9,  8,  7, 12, 11, 10]),\n",
       " 'split0_test_score': array([-0.5892449 , -0.59013377, -0.58904676, -0.58262428, -0.58031286,\n",
       "        -0.58084282, -0.59389733, -0.5936689 , -0.59154331, -0.61925128,\n",
       "        -0.61288703, -0.60894958]),\n",
       " 'split0_train_score': array([-0.55875226, -0.55985588, -0.56083729, -0.46494002, -0.47335579,\n",
       "        -0.48028504, -0.32258111, -0.35448239, -0.37433042, -0.17838817,\n",
       "        -0.23480585, -0.26797843]),\n",
       " 'split1_test_score': array([-0.59466973, -0.59501965, -0.59467034, -0.59184177, -0.59104339,\n",
       "        -0.59109635, -0.60387368, -0.6022403 , -0.59817704, -0.63921621,\n",
       "        -0.62472302, -0.62089215]),\n",
       " 'split1_train_score': array([-0.55581464, -0.5570608 , -0.55808877, -0.46105021, -0.47036484,\n",
       "        -0.47661754, -0.31895185, -0.35094927, -0.37063188, -0.17455118,\n",
       "        -0.22839129, -0.2660666 ]),\n",
       " 'split2_test_score': array([-0.59653524, -0.59639524, -0.59509482, -0.591487  , -0.59078388,\n",
       "        -0.59015817, -0.60471616, -0.59904132, -0.60001313, -0.63694585,\n",
       "        -0.62154312, -0.61718315]),\n",
       " 'split2_train_score': array([-0.5568092 , -0.55728467, -0.55810203, -0.46153599, -0.47078398,\n",
       "        -0.47666763, -0.31817004, -0.3508376 , -0.37260707, -0.17814619,\n",
       "        -0.23268101, -0.26495781]),\n",
       " 'split3_test_score': array([-0.59804914, -0.5992678 , -0.59944017, -0.59489087, -0.59472401,\n",
       "        -0.59416523, -0.6085894 , -0.60455027, -0.60390414, -0.64303202,\n",
       "        -0.63187688, -0.62514574]),\n",
       " 'split3_train_score': array([-0.55429522, -0.55627275, -0.55682348, -0.46049956, -0.470265  ,\n",
       "        -0.47651104, -0.31854738, -0.35091709, -0.37064765, -0.17747491,\n",
       "        -0.22632094, -0.26592155]),\n",
       " 'std_fit_time': array([1.41447827, 0.31739826, 1.75657424, 1.58777651, 0.89000575,\n",
       "        0.19090769, 0.31475951, 0.93867825, 0.77432942, 0.46177977,\n",
       "        1.25026408, 0.81069083]),\n",
       " 'std_score_time': array([0.06899921, 0.02280443, 0.03071902, 1.18255467, 0.27926472,\n",
       "        0.34587191, 1.56953658, 1.36342115, 0.94173266, 1.85334762,\n",
       "        1.81188823, 1.85448505]),\n",
       " 'std_test_score': array([0.00332876, 0.00330437, 0.00369157, 0.00457575, 0.00537135,\n",
       "        0.00497346, 0.00542215, 0.00408243, 0.00447109, 0.00913109,\n",
       "        0.00681778, 0.00595816]),\n",
       " 'std_train_score': array([0.00161803, 0.0013453 , 0.00146592, 0.00173294, 0.00126412,\n",
       "        0.00159722, 0.00176453, 0.00155118, 0.00154015, 0.00153171,\n",
       "        0.00336149, 0.00109514])}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GSearch_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.589065 using {'max_depth': 5, 'min_child_weight': 5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\Dell\\Documents\\anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt8VfWd7//XZ19yTyCAXBPEIgqKSBURb4gXIKFWf479WXXsUY/VmbEep3Meba2dTq091jqnnQ69MI621Tqtt7ZTHeqQEMALlhnlJiiiKN4ggIAhISHXffmcP9bayc7OTvZOyL4k+Twfjzyy117ftfZnL8h+7++6fUVVMcYYY/riyXQBxhhjsp+FhTHGmIQsLIwxxiRkYWGMMSYhCwtjjDEJWVgYY4xJyMLCGGNMQhYWxhhjErKwMMYYk5Av0wUMlnHjxum0adMyXYYxxgwpW7Zs+VRVT0jUbtiExbRp09i8eXOmyzDGmCFFRD5Opp3thjLGGJOQhYUxxpiELCyMMcYkZGFhjDEmIQsLY4wxCVlYGGOMScjCwhhjTEIjPixUlbpHH6Nt17vYELPGGBPfiA+LQG0th370Iz686io+uOLzHP7Zz2l///1Ml2WMMVlFhsu36Xnz5ulAr+AOfvopTWvW0LiqipbNm0GV3BkzKFlWSXFFBbknnTTI1RpjTHYQkS2qOi9hOwuL7gKHDtG0uobG6mpat2wBIHfWLEoqKiiprCBn6tTjfg1jjMkWFhaDIPDJJzStXk3jqipat28HIO/0050ex9IKcsqmDOrrGWNMullYDLLAvn00Vq+msaqKth07AMg7cw4llZWULF2Kf9KklL22McakioVFCnXs3UtjVTWN1VW073wbgPyzzqKkooLipUvxTxifljqMMeZ4WVikSfuHH3buqmp/910QoeDssyleVknJkiX4xo1Le03GGJMsC4sMaH//fafHUVVFx/vvg8dDwfz5bo9jCb7S0ozWZ4wxsSwsMkhVaX/vPRqrqmhaVUXHxx+D10vhggWUVFZQfPnleEePznSZxhhjYZEtVJX2d97p7HEE9u4Fn4/C88+jpHIZxZddirekJNNlGmNGKAuLLKSqtL21k8aqVTRVVRPYvx/x+ym88EJKKisouvRSvEVFmS7TGDOCWFhkOVWl7Y033LOqqgl+8gmSk0PhwosoqaykeNEiPIWFmS7TGDPMWVgMIRoO07ptG41V1TRVVxM8fBjJy6Po4ospqayk6OKFePLzM12mMWYYsrAYojQUonXrVhqrqmhcXUOorg4pKKB40SKKKysoWrgQT25upss0xgwTFhbDgIZCtGza5PQ4amoI1dfjKSyk6NJLKamspPDCC/Dk5GS6TGPMEGZhMcxoMEjza685p+OuWUv46FE8xcUUX3YZJcsqKVywALHgMMb0k4XFMKYdHTS/+iqNq6poWreOcFMTnlGjKF58OSUVlRQuOBfx+TJdpjFmCMiKsBCRCuAngBf4pao+GKfNtcB3AQW2q+oNInIi8Ed3OT/wM1X9175eaySFRbRwRwfNf95AY3UVx9a9QLi5GW9pKcWLF1OyrJKCc85BvN5Ml2mMyVIZDwsR8QLvAouBWmATcL2q7oxqMwP4HXCpqtaLyHhVPSQiOW5t7SJSBOwAzlfV/b293kgNi2jh9naaX3nF6XG89BLa0oJ37FhKli6huKKCgrPPtuAwxnSTbFikcl/FfGC3qn7gFvQ0cBWwM6rNbcAKVa0HUNVD7u+OqDa52PCvSfHk5lJ8+eUUX3454dZWjr28nsaqKhr++Cz1Tz6F74QTKF66lJJlleTPnYt4bLMaY5KTyrCYAuyNmq4Fzo1pcwqAiGzA2eX0XVWtdp8rB/4TOBn4erxehYjcDtwOMNVGsOvGk59PScVSSiqWEm5upumll2iqrqbhd7+j/re/xTdxIiVucOTNmYOIZLpkY0wWS+VuqP8fWKqqX3anvwTMV9X/FdXmeSAAXAuUAa8As1W1IarNZOA54POqerC317PdUMkJHTvGsRdfpHFVFc1//jMaCOCfPJniygpKKirJm326BYcxI0g27IaqBcqjpsuA2N5BLfCqqgaAD0VkFzAD5/gGAKq6X0TeAi4C/pDCekcEb1ERoz7/eUZ9/vOEGhtpWvcCjdVVHHn83zjyq0fxl5c7440vqyR35kwLDmMMkNqehQ/nAPdlwD6cALhBVd+KalOBc9D7JhEZB7wOzAXygTpVbRWRUuA14BpVfbO317OexfEJNTTQtHYtjVXVNL/6KoRC5Eyb5vQ4KivJnTHDgsOYYSjjZ0O5RSwDluMcj3hUVb8vIt8DNqvqSnE+ff4JqABCwPdV9WkRWew+r4AAP1fVR/p6LQuLwRM8coSmNWtprKqiZeNGCIfJmT7dGW+8soLc6dMzXaIxZpBkRVikk4VFagQ//ZTGmhqaqqpp2bwZVMk95RRK3B5HzrRpmS7RGHMcLCzMoAscPERTTQ2NVVW0bt0KQO6sWZ09jpzy8gRrMMZkGwsLk1KBAwdoXL2axqoq2ra/AUDe7NlOcFQsxT9lSoYrNMYkw8LCpE1H7T6aVlfTWFVN244dAOSfeaZ7Om4F/okTM1yhMaY3FhYmIzr27Okc/a/97bcByD/rLGf0v6VL8I8fn+EKjTHRLCxMxrV/+CFN1U6Po/3dd0GEgnnznB7HkiX4xo3LdInGjHgWFiartO/e7fQ4qqro+OAD8HgoOHc+JRWVFC9ZjK+0NNMlGjMiWViYrKSqtL/7Ho3VVTStqqLj44/B66VwwQJKllVSfNlleEePznSZxowYFhYm66kq7e+8Q+OqKhqrqgjU1oLfT+H55zk9jssvw1tcnOkyjRnWLCzMkKKqtO14i8aqKhqrqwjuP4D4/RReeCElyyopuuQSvEVFmS7TmGHHwsIMWapK2/btnWdVBQ8eRHJyKLp4ISWVlRQtWoSnoCDTZRozLFhYmGFBw2Fat21zdlWtriZ0+FMkL4+iRYsoqaig6OKFePLzM12mMUOWhYUZdjQUomXLFhqrqmhaXUPoyBGkoIDiRYsoWVZJ4UUX4cnNzXSZxgwpFhZmWNNgkJZNm2isqqappoZQQwOewkKKLruUkopKCi+8AE9OTqbLNCbrWViYEUMDAZpf20hj1Sqa1q4jfPQonuJiii+/nJLKCgrPOw/x+zNdpjFZycLCjEja0UHzf/+30+NYu5bwsWN4R42iaPHllFRWUnjuuYgvlQNEGjO0WFiYES/c0UHzn/9MY1U1x9atI9zSgre0lOIlSyiprKDgnHMQrzfTZRqTURYWxkQJt7Vx7JVXaKqqounFl9DWVrzjxlGyZDEllZXkn3024vFkukxj0s7CwphehFtbOfbyy06P4+WX0bY2fCecQHGFM/pf/twzLTjMiGFhYUwSws3NNL30Eo1VVTSvfwXt6MA3aRIlS5dSUllB3pw5OEPFGzM8WVgY00+hY8c49sILNK6q4tiGDRAI4J882bmleuUy8k4/zYLDDDsWFsYch1BjI01r19FYXUXzf/03BIP4p06lpKKCkmWV5J56qgWHGRYsLIwZJMH6eo6tW0fjqiqaX3sNQiFypk1zbqleUUHeKadkukRjBszCwpgUCB45QlPNGhqrqmjZtAnCYXJOnk5JRaXT4/jMZzJdojH9YmFhTIoFDx+msaaGpqpqWrZsAVVyTzmFkmWVlFRUkDNtWqZLNCYhCwtj0ihw8CBNq2torKqi9fXXAcg9bRYllW5wlJdnuEJj4rOwMCZDAgcO0Fi9msbqKtq2vwGAb+JEcsrK8E+dSk55Gf6ycnKmluMvL8dbWmoHy03GWFgYkwU6avfRVFND+65ddNTWEti7l+ChQ93aeAoL8ZeXk1Ne7v4uw1/uhsrkyXYTRJNSyYaF3VHNmBTKKZvC2P95S7fnwq2tBPbto2PPXgK1e53fe/fS/sEHzhXlHR1djT0e/JMmdQ+TqeWdPRNvSUma35EZqSwsjEkzT34+uSefTO7JJ/eYp+EwwcOHCezZQ8feWjr27iGw1+mRNK1bR+jIke7rGjXK3b1VTk5ZufPbDRbfxIl2o0QzaCwsjMki4vHgnzAB/4QJFJxzTo/5oWPNTm9k714CUWHStnMnTWvWQjDY1djvxz95EjnlU/GXl0X9dnom3qLCNL4zM9RZWBgzhHiLCvHOnEnezJk95mkoRODAJ11hsmcvHbXO79Y33yR89Gj3dY0Z03PXVrlzEN53wgl2M0XTjYWFMcOEeL3klE0hp2wKhQsW9JgfOnqUjr213Y6TdOzdS+vrr9O4ahWEw13rys3FX1bm7OKKOU7iLyvDk5eXzrdmsoCFhTEjhHfUKPJHjSJ/9uk95mkgQGD/fidM9u7p9rtl0ybCLS3d2vvGj3dCJHK8pPNMrnK8Y8faqcDDkIWFMQbx+8k58URyTjwRuKDbPFUlVF/v9ERizuBqfu01gv/xH93XVVDQ1SMpL3eOk0ydir+sDP+UKXhyctL4zsxgSWlYiEgF8BPAC/xSVR+M0+Za4LuAAttV9QYRmQs8BJQAIeD7qvpMKms1xsQnIvjGjME3Zgz5Z57ZY364vd09Fdg9cysSJns+pnnDBrStLXpl+CZN7DpzK+riRH9ZGd7Ro61XkqVSFhYi4gVWAIuBWmCTiKxU1Z1RbWYA9wAXqGq9iIx3Z7UA/0NV3xORycAWEVmtqg2pqtcYMzCe3FxyP/OZuDdRVFXnVOC9Pc/gOvbSy4Q+/bT7uoqLO8/cir3S3T9pEuKznSGZksotPx/YraofAIjI08BVwM6oNrcBK1S1HkBVD7m/3400UNX9InIIOAGwsDBmCBER/OPH4x8/noKzz+4xP9zS0nXQPeoMrvZdu2h64QUIBLoae734J0/ufoV75PfUqXiLitL4zkaeVIbFFGBv1HQtcG5Mm1MARGQDzq6q76pqdXQDEZkP5ADvp65UY0wmeAoKyDv1FPJO7TkmiIZCBA8ejHsGV9vq1YQaun939I4e3euV7r7x4+0CxeOUyrCIt+Mx9kZUPmAGsAgoA14RkdmR3U0iMgn4DXCTqoZjlkVEbgduB5g6dergVW6MyThxexL+yZPh3Pk95oeamtzw6H4GV+ubb9K4ejWEQl3r8vvxT5nS40r3yLUlnoKCdL61ISmVYVELRN+XuQzYH6fNq6oaAD4UkV044bFJREqA/wS+raqvxnsBVX0EeAScGwkOcv3GmCzmLS7Ge9pp5J12Wo95GgwSOHCgxxlcHbV7ad36OuFjx7qva9y4rjO3oq90Ly93LlC0g+4pDYtNwAwROQnYB1wH3BDT5jngeuDXIjIOZ7fUByKSAzwL/Juq/n6gBQQCAWpra2mLPhvDDGt5eXmUlZXhtzu1jmji83XeI6vw/O7zVJVQQwOB2trOM7giV7q3bN5M45+eh6i7cUteXveD7WVRpwNPmYInNzfN7y4zUhYWqhoUkTuB1TjHIx5V1bdE5HvAZlVd6c5bIiI7cU6R/bqq1onIjcBCYKyI3Oyu8mZV3dafGmpraykuLmbatGn2zWAEUFXq6uqora3lpJNOynQ5JkuJCL7SUnylpeSfcUaP+eGODgL79sWcweVeV/Lqq2hra/TK8E2YEP9K92E2VsmwHs/i7bffZubMmcPmH8skpqq88847zJo1K9OlmGFIVQnV1XXt2uo8g6uWwJ49BA8f7ta+a6ySOGdwZclYJTaehcuCYmSxf2+TSiKCb9w4fOPGwVmf7TG/17FK3v+AYy+vT36skvIyvKNGpfGdJTbsw8IYY9JlIGOVdOzd06+xSvxl5fgnpX+sEguLFGtoaODJJ5/kjjvu6Peyy5cv5/bbb6cgC0/rW7RoET/60Y+YNy9h77WH5557jlNOOYXT3LNYkllXW1sbCxcupL29nWAwyBe+8AXuu+++AddvTLr1a6ySyO3lkxyrJG/WLEqvvTal9VtYpFhDQwP/8i//MuCwuPHGG7MyLI7Hc889xxVXXNEZFsnIzc3lhRdeoKioiEAgwIUXXkhlZSUL4tyK25ihqM+xSoJBAp8cdHdtdT+Dq/XNN+l4b7eFxWC5709vsXN/46Cu87TJJdz7+Z63e472zW9+k/fff5+5c+eyePFixo8fz+9+9zva29u5+uqrue+++2hububaa6+ltraWUCjEP/zDP3Dw4EH279/PJZdcwrhx43jxxRfjrr+oqIivfOUrrF27ltLSUh544AG+8Y1vsGfPHpYvX86VV17JRx99xJe+9CWam5sB+PnPf87555/Ps88+y4oVK1izZg2ffPIJF198MevXr2fixIk9Xqe1tZVbbrmFnTt3MmvWLFqjzgipqanh3nvvpb29nenTp/PYY49RVFTEtGnT+OIXv9hZ+5NPPsmhQ4dYuXIlL7/8Mvfffz///u//DsDvf/977rjjDhoaGvjVr37FRRdd1O31RYQi93YOgUCAQCBgxyfMiCE+X59jlYTb21NeQ7+GwhIRj3uxnEnSgw8+yPTp09m2bRuLFy/mvffeY+PGjWzbto0tW7awfv16qqurmTx5Mtu3b2fHjh1UVFRw1113MXnyZF588cVegwKgubmZRYsWsWXLFoqLi/n2t7/NmjVrePbZZ/nOd74DwPjx41mzZg1bt27lmWee4a677gLg6quvZuLEiaxYsYLbbruN++67L25QADz00EMUFBTwxhtv8Pd///ds2bIFgE8//ZT777+ftWvXsnXrVubNm8ePf/zjzuVKSkrYuHEjd955J1/96lc5//zzufLKK/nhD3/Itm3bmD59OgDBYJCNGzeyfPnyzt1L+/fvZ9myZZ3rCoVCzJ07l/Hjx7N48WLOPTf27jHGjEzpuNYjYc9CRJ4E/hrnOogtwCgR+bGq/jDVxQ2mRD2AdKipqaGmpobPftY5i+LYsWO89957XHTRRXzta1/j7rvv5oorrujxrbovOTk5VFRUAHDGGWeQm5uL3+/njDPO4KOPPgKcb+J33nkn27Ztw+v18u67nfdp5Gc/+xmzZ89mwYIFXH/99b2+zvr16ztDZs6cOcyZMweAV199lZ07d3LBBc4YCB0dHZx33nmdy0XWef311/N3f/d3va7/L/7iLwA4++yzO+uePHkyq1at6mzj9XrZtm0bDQ0NXH311ezYsYPZs2cntZ2MMccnmd1Qp6lqo4j8JbAKuBsnNIZUWGQDVeWee+7hr/7qr3rM27JlC6tWreKee+5hyZIlnb2CRPx+f+fuGI/HQ677DcPj8RB0D4j98z//MxMmTGD79u2Ew2HyoobE3LdvHx6Ph4MHDxIOh/H0Me5yvN0+qsrixYt56qmnEi7T126jSN1er7ez7t6MHj2aRYsWUV1dbWFhTJoksxvKLyJ+4P8D/sO9j9PwuJIvDYqLi2lqagJg6dKlPProoxxz70uzb98+Dh06xP79+ykoKODGG2/ka1/7Glu3bu2x7PE4evQokyZNwuPx8Jvf/IaQe4O1YDDILbfcwpNPPsmsWbO67T6KtXDhQp544gkAduzYwRtvvAHAggUL2LBhA7t37wagpaWlW8/lmWee6fwd6XEM5H0dPnyYBvcuo62traxdu5aZcQ4EGmNSI5mexcPAR8B2YL2InAgM7pHiYWzs2LFccMEFzJ49m8rKSm644YbOD82ioiJ++9vfsnv3br7+9a/j8Xjw+/089NBDANx+++1UVlYyadKkPo9bJHLHHXdwzTXX8Pvf/55LLrmEwsJCAB544AEuuugiLrroIubOncs555zD5z73ubhXP//N3/wNt9xyC3PmzGHu3LnMn+/cBfSEE07g17/+Nddffz3t7kG2+++/n1NOcW453d7ezrnnnks4HO7sfVx33XXcdttt/PSnP+UPf/hDr3Xv37+fL3/5y6xatYoDBw5w0003EQqFCIfDXHvttVxxxRUD3ibGmP4Z0O0+RMSnqn3vK0iz3m73Ybd9yJxp06axefNmxo0bl9bXtX93Y5KX7O0+Eu6GEpG/FZEScfxKRLYClw5KlcYYY4aEZHZD/U9V/YmILMUZ2vQW4DGgJqWVmW7OPffczt08Eb/5zW84I85dM4/H6tWrufvuu7s9d9JJJ/Hss8/2e12Rs5qMMUNfMmEROYVlGfCYqm4Xuxoq7V577bW0vM7SpUtZunRpWl7LGDN0JHM21BYRqcEJi9UiUgz0GOLUGGPM8JVMz+JWYC7wgaq2iMhYnF1RxhhjRoiEYaGqYREpA25w9z69rKp/SnllxhhjskYyZ0M9CPwtsNP9uUtEfpDqwowxxmSPZI5ZLAMWq+qjqvooUAF8LrVlDR+RW5QPxPLly2lpaRnkigbHokWLiL2uJVnPPfccO3fu7Pe6pk2bxhlnnMHcuXMHNI6GMWbgkr3r7Oiox9k11l+WG65hcTxiw6I/XnzxRbZt2zbgoDLGDEwyB7h/ALwuIi/inEa7ELgnpVWlQtU34ZM3B3edE8+Aygf7bGLjWQzOeBbGmMxK2LNQ1aeABcAf3Z/zgPUprmvYsPEsBm88CxFhyZIlnH322TzyyCPH8a9ijOmvpEbKU9UDwMrItIjsAaamqqiUSNADSAcbz+L4xrPYsGEDkydP5tChQyxevJiZM2eycOHCpLaTMeb4DHRYVbuCewBsPIvjG89i8uTJgNNTuvrqq9m4caOFhTFp0q9hVaPYeBZJsvEsBmc8i+bm5s5lmpubqampsYGPjEmjXnsWIvIz4oeC0P3sKNMHG89icMazOHjwIFdffTXghNwNN9zQufvNGJN6vY5nISI39bWgqj6ekooGyMazyD42noUx2S/Z8Sx67VlkWxgYY4zJnIEe4DZpZuNZGGMyycJiiLDxLIwxmTTQs6GMMcaMIAl7FiLy0zhPHwU2q+p/DH5Jxhhjsk0yPYs8nMGP3nN/5gBjgFtFZHkKazPGGJMlkjlmcTJwqaoGAUTkIaAGWAwM8p35jDHGZKNkehZTgMKo6UJgsqqGgPb4izhEpEJEdonIbhH5Zi9trhWRnSLylog8GfV8tYg0iMjzSdSYtYbrLcrTPZ7Frl27mDt3budPSUkJy5dbx9aYdEkmLP4vsE1EHhORXwOvAz8SkUJgbW8LiYgXWAFUAqcB14vIaTFtZuDc7vwCVT0d+GrU7B8CX+rHe8lKwzUsjsdAxrM49dRT2bZtW+fdegsKCjqv6DbGpF4yY3D/SkRWAfNxbvXxLVXd787+eh+Lzgd2q+oHACLyNHAVztCsEbcBK1S13n2tQ1Gvu05EFvXjvfTpHzf+I+8ceWewVgfAzDEzuXv+3X22sfEsBn88i3Xr1jF9+nROPPHEvv+BjDGDJtlTZ88BLgIuBM5OcpkpwN6o6Vr3uWinAKeIyAYReVVEht3Nfmw8i8EbzyLi6aef7vN26saYwZfMqbMP4oTFE+5Td4nI+aqaaLS8ePejjr0RlQ+YASwCyoBXRGS2qjYkqsut7XbgdoCpU/seXiNRDyAdbDyL4xvPIrL+lStX8oMf/CDhtjHGDJ5kzoZaBsxV1TCAiDyOc9wiUVjUAuVR02XA/jhtXlXVAPChiOzCCY9NSdSFqj4CPALOjQSTWSaTbDyL4xvPAqCqqoqzzjqLCRMm9NrGGDP4kt0NFX1L8lFJLrMJmCEiJ4lIDnAdUaPtuZ4DLgEQkXE4u6U+SHL9Q4KNZzE441lEPPXUU7YLypgMSKZn8QPgdRF5EWfX0kIS9ypQ1aCI3AmsBrzAo6r6loh8D+fq75XuvCUishMIAV9X1ToAEXkFmAkUiUgtcKuqru7/W8wsG89icMazACeI1qxZw8MPPzzgbWGMGZhex7Po1khkEs5xCwFeAzxRZ0RlBRvPIvvYeBbGZL/jHs8imqoeIGoXkojsAfo+omyMMWbYGOgtyns/UmlSwsazMMZk0kDDIuvPPBpubDwLY0wm9RoWIvIz4oeC0P3sKGOMMcNcXz2Lvu7sNrA7yBljjBmSeg0LVX08nYUYY4zJXsnc7mMe8PfAidHtVXVOCusyxhiTRZK5gvsJ4DHgGuDzUT8mCcP1FuXpHs8C4Cc/+QmzZ8/m9NNPt7EsjEmzZMLisKquVNUPVfXjyE/KKxsmhmtYHI+BjGexY8cOfvGLX7Bx40a2b9/O888/z3vvvZeiCo0xsZI5dfZeEfklsI6okfFU9Y8pqyoFPnngAdrfHtzxLHJnzWTit77VZxsbz2JwxrN4++23WbBgAQUFBQBcfPHFPPvss3zjG99I8l/LGHM8kulZ3ALMBSro2gV1RSqLGk5sPIvBGc9i9uzZrF+/nrq6OlpaWli1ahV79+7FGJMeyfQszlTVwb1MOAMS9QDSwcazGPh4FrNmzeLuu+9m8eLFFBUVceaZZ+LzDfSaUmNMfyXz1/aqiJymqv3byWx6sPEsjm88i1tvvZVbb70VgG9961uUlZX1uj5jzOBKZjfUhcA2EdklIm+IyJsi8kaqCxsubDyLwRvP4tAhZ4j2PXv28Mc//tHGtTAmjZLpWQy7cbHTycazGLzxLK655hrq6urw+/2sWLGC0tLSAW8TY0z/JDWexVBg41lkHxvPwpjsl+x4FskOq2qMMWYEs9NJhggbz8IYk0nDPixUtc+zcIYKG88iOcNlt6ox2WZY74bKy8ujrq7OPkBGCFWlrq6u26nBxpjBMax7FmVlZdTW1nL48OFMl2LSJC8vz66/MCYFhnVY+P1+TjrppEyXYYwxQ96w3g1ljDFmcFhYGGOMScjCwhhjTELD+phFMkLhENsPb6c0r5QxeWMozinGI5ahxhgTbcSHRUN7AzdV39Q57RUvpXmlTnjkjul8HJkekz+G0lwnWErzShmVO8rCxRgz7I34sCjKKeLhyx+mrq2O+rZ66tvrqW+r50jbEerb6nn7yNscaT1CUyD+XVI94mF07mhKc0s7eyfRv2NDZ3TuaHyeEb/ZjTFDzIj/1Mr15nL+lPMTtguEAj2CpL7dedw53VbPu/XvUt9ez9H2o3HXIwijckc54ZEbEyp5Y7qm3Xmj80bj9/gH+20bY0y/jPiwSJbf62d8wXjGF4xPqn0wHKShvaFbuBxpO9IjcD48+iFbD22lvq0eJf6V5sU5xU6I5Pbdc4k8l+PNGcy3bowxFhap4vP4GJebi1MbAAATCklEQVQ/jnH5yd2eOxQOcbTjaPeeS1s9R9qPdHtuT9Meth/eTkN7AyENxV1Xkb8o7i6wbj2XqHl5Prs9hjGmbxYWWcLr8XZ+mE9nesL2YQ3T1NHUbTdY7O6x+rZ6DjQf4K26t6hvrycYjj9cab4vP+4usOiQKc0t7Ty4X+AvGOy3b4zJchYWQ5RHPIzKHcWo3FGcNCrxLU1UlaZAU1ePJSpY6lrrOnePHW45zK4ju6hvq6cj3BF3XXnevJ5nifWxe6zQXzgs7vxrzEhmYTFCiAglOSWU5JRwYsmJCdurKi3Blh4H8OP1Xj5o+ID6tnraQm1x15XjyekRJLG9l7F5YzsfF/uLLVyMyTIWFiYuEaHQX0ihv5Dy4vKklmkJtPQ8Y6wt6owxd97HjR9T31ZPS7Al7np8Hl/nqcjdjrXEnJ4c6b2U5JbYtS7GpFhKw0JEKoCfAF7gl6r6YJw21wLfBRTYrqo3uM/fBHzbbXa/qj6eylrN8SvwF1DgL2BK0ZSk2rcF22hob+i6xqWXnsuOT3dQ31bPscCxuOvxite51iU6SOKcOdZ5IWXOKLwe72C+dWOGvZSFhYh4gRXAYqAW2CQiK1V1Z1SbGcA9wAWqWi8i493nxwD3AvNwQmSLu2x9quo16Zfny2OibyITCycm1b4j1NEjSLr1XNx5u47s4kjbERo7GuOuxyMeRuWM6vP6luh5diGlMantWcwHdqvqBwAi8jRwFbAzqs1twIpICKjqIff5pcAaVT3iLrsGqACeSmG9JsvleHOYUDiBCYUTkmofCAc42n602wH86JCJhM7uht3UtzkXUvZ2rcuo3FE9gqQ0t5Sx+WN77h7LLcXvtQspzfCSyrCYAuyNmq4Fzo1pcwqAiGzA2VX1XVWt7mXZ5PZtGOPye/z9vtYlciFldO8ltufycePHvH7odRraGwhrOO66iv3F3QKkOKeYPG8e+b588v35XY/j/OT58npM2zEZk2mpDIt4p7PEfm3zATOARUAZ8IqIzE5yWUTkduB2gKlTpx5Prcbg9XgZmz+Wsfljk2of1jCN7Y09DuDHXlRZe6yWlkALrcHWzp/+ioRLdJDEhkpfYVPgK3AexwksCyOTjFSGRS0QfRpNGbA/TptXVTUAfCgiu3DCoxYnQKKXfSn2BVT1EeARgHnz5sXff2BMinjEw+i80YzOG81n+EzSy6kqbaG2zuBoC7Z1C5JEz0fPq2+r50DwQPd5vZzC3JduQdJHKBX4CpIKrOh5ed48OxV6GEhlWGwCZojIScA+4Drghpg2zwHXA78WkXE4u6U+AN4HHhCRUrfdEpwD4cYMeSLS+YGaCmENd4ZJW6iN1kD3IGkJttAaaE0qsOra6rrNawu2DTiMOkMkqncTG1JJ95Ci5uV6cy2M0iBlYaGqQRG5E1iNczziUVV9S0S+B2xW1ZXuvCUishMIAV9X1ToAEfk/OIED8L3IwW5jTN884uk8jTkVImHUEmzpESSJekKxgXW45XCP0GoPtferHkGS2iWX583rDKh8b378wPJHzXOXtzByiOrw2Hszb9483bx5c6bLMMYcp1A41C1A4gaOG1a9zesrtHq7jU1vPOLpccxoILvjYpePTOd4cjIaRiKyRVXnJWpnJ48bY7KK1+Ol0OPcPSAVguFg5+601kCr00OKhFOgq/eTzHGkppYmWoOt3XpZgXCgX/V4xNNn7yaZExvG5Y9j7vi5KdleERYWxpgRxefxUZRTRBFFkILDRsFwcOAnLkQdY2oLtXG05WhXiIWc5+PdPXrOuDk88bknBv/NRLGwMMaYQeTz+CjOKaY4pzgl6w+EAz12u3kl9bevsbAwxpghxO/x48/xpyyMemNX4hhjjEnIwsIYY0xCFhbGGGMSsrAwxhiTkIWFMcaYhCwsjDHGJGRhYYwxJiELC2OMMQlZWBhjjEnIwsIYY0xCdrsPY3BGrwuFlaD7EwopwXC467l+TofCSiDUNS2A3+vB5xV8HsHniTx2fvu9gtfjwecR/F4PXo/znM/rcdtHPfYKfo8Hj8fGWDDpY2Fh4gpHf+iFw+6HZ88PwXjTwXCYYEijngsTSDDd9YHbfTqyPud1YpYJRX249zHduY6o+cGomiNthhqPEBU6XWESCZtIqEQHT+djT1fwdAWSxw0tZx09Aiqy3thAiwm+yHJ+j7h19Fxvjxrd35HlbbCh7GNhkYS+vnUGYqdjPuCS/RYaDIWjPtgST4dC7od4TE1d6+s53fnBH/NB3LV813Qmx8TyRT5koj6s4k1HPmwiH3xej5Dn93Sbjp3v80bW5T4X/U0/dtrb9breqA/SyHTkg83r6f5BGZmOfJh63A++SKgGov5tgqGuf9tgVIgGQuFu/2adQei27bauqOWj20X/34xeb1sgTDAccpYLdf8/0bXeyDqddaU7Szv/vWPCpkeguT0yf49Aixd8vQVa995dvDCNBF/Xa0attx+9Q+8QDsIRHxZHmju4asWf3Q/frg/uziDI8LfOyLfHrg++rv+Qkf+MsR9OXR+GHvL8Mct4pesPJHY68kcQM935H90b+7q9Tyf6oI9Md334Dv0/puEs0tOMDp7oQOoKtK7fscHXe6BFArCr1xk3TKNeJ16gBUNKczDYbV2dYej+fUevK1JjukXvSvRFB1S3kOq+y7GzF9ZL7/DEsYV85ZKTU1t3Stc+BOT4PJxz4pi0f+vs7Vto9LTPI7Zf2mQFj0fI8Qg5w/CcmOjg6S34gj16Xol7h8FQH8EXp3cY+YLaFWjdg601EBOSUfNnTerfuOUDMeLDoijXx4+/mNrhCI0x2cvZpZn6wYOGuuH3NcEYY8ygs7AwxhiTkIWFMcaYhCwsjDHGJGRhYYwxJiELC2OMMQlZWBhjjEnIwsIYY0xCFhbGGGMSsrAwxhiTkIWFMcaYhCwsjDHGJGRhYYwxJiELC2OMMQmlNCxEpEJEdonIbhH5Zpz5N4vIYRHZ5v58OWreP4rIDvfni6ms0xhjTN9SNp6FiHiBFcBioBbYJCIrVXVnTNNnVPXOmGU/B5wFzAVygZdFpEpVG1NVrzHGmN6lcvCj+cBuVf0AQESeBq4CYsMintOAl1U1CARFZDtQAfxu0Ktsa4Tf3wy+XOfHmwu+HPDlgTcn6rnIfPe5bvPd9r4ct23048i6csFje/2MMUNTKsNiCrA3aroWODdOu2tEZCHwLvB3qroX2A7cKyI/BgqAS0guZPovHIS2oxBsh1C7+7sDgm0Q7HCeCwcH57U8vl6CJyqQegRPXyEWadvLuqKXi33OO+IHSTTG9EMqPzHiDR4dOzr6n4CnVLVdRP4aeBy4VFVrROQc4L+Aw8B/Az0+sUXkduB2gKlTpw6syoIxcNu6vtuEQ1Fh4gZJqKN7wHSGTHvfwdNtfuy6OqDjGLTUxVnOnQ4HBvY+Y4knQe+pl95Rj+A5jp5W5DmvH8TGGjcmm6UyLGqB8qjpMmB/dANVrYua/AXwj1Hzvg98H0BEngTei30BVX0EeARg3rx5sUE0eDxeyCnA6eRkWDjsBEmPkGobvMCKrCvQCq0NfQfloJDee0999Y6S2l2YZE8rel0WXMb0kMqw2ATMEJGTgH3AdcAN0Q1EZJKqHnAnrwTedp/3AqNVtU5E5gBzgJoU1jp0eDzgyQN/XqYrAdWYkGnvHibRIRNqj99TShRY0etqb+q7Z9ej4zpA3ni9pz52DSbT0/L43BCSPn57uoIq6bbx5tE1LZ4E65E4belH23ht6EfbeOtN5v0nWK8F/qBLWVioalBE7gRWA17gUVV9S0S+B2xW1ZXAXSJyJc4upiPAze7ifuAVcf7BG4Eb3YPdJpuIdH1IZpqqc2wpYa8qXmD1tkuxj95Ya33P14peTsOZ3iIG6H9geZILy14Du7dl+wr32GWTaRuz3hNOhc/9U0q3ZEqPcqrqKmBVzHPfiXp8D3BPnOXacM6IMiY5Is6xD68fcosyXQ2Egt1DRENOoKG9/MYJmD7bRP+Obks/2vb2m360jV0v/ay3t/ffn3qj2yZ6/267pF47UdvY9SZbZ2/r7WXbJVWDu2w45PykmJ0SY0wqeH3OT05hpisxZlDYif/GGGMSsrAwxhiTkIWFMcaYhCwsjDHGJGRhYYwxJiELC2OMMQlZWBhjjEnIwsIYY0xCopErAYc4ETkMfHwcqxgHfDpI5Qwmq6t/rK7+sbr6ZzjWdaKqnpCo0bAJi+MlIptVdV6m64hldfWP1dU/Vlf/jOS6bDeUMcaYhCwsjDHGJGRh0eWRTBfQC6urf6yu/rG6+mfE1mXHLIwxxiRkPQtjjDEJjaiwEJFHReSQiOzoZb6IyE9FZLeIvCEiZ2VJXYtE5KiIbHN/vhOvXQrqKheRF0XkbRF5S0T+Nk6btG+zJOtK+zYTkTwR2Sgi29267ovTJldEnnG312siMi1L6rpZRA5Hba8vp7quqNf2isjrIvJ8nHlp315J1JTJbfWRiLzpvu7mOPNT9/eoqiPmB1gInAXs6GX+MqAKZ4DDBcBrWVLXIuD5DGyvScBZ7uNi4F3gtExvsyTrSvs2c7dBkfvYD7wGLIhpcwfwr+7j64BnsqSum4Gfp/v/mPva/xt4Mt6/Vya2VxI1ZXJbfQSM62N+yv4eR1TPQlXX44z13ZurgH9Tx6vAaBGZlAV1ZYSqHlDVre7jJuBtYEpMs7RvsyTrSjt3GxxzJ/3uT+xBwauAx93HfwAuE3ew+QzXlREiUgZ8DvhlL03Svr2SqCmbpezvcUSFRRKmAHujpmvJgg8h13nuboQqETk93S/udv8/i/OtNFpGt1kfdUEGtpm7+2IbcAhYo6q9bi9VDQJHgbFZUBfANe6uiz+ISHmqa3ItB74BhHuZn4ntlagmyMy2Aifka0Rki4jcHmd+yv4eLSy6i/eNJRu+gW3FuST/TOBnwHPpfHERKQL+HfiqqjbGzo6zSFq2WYK6MrLNVDWkqnOBMmC+iMyOaZKR7ZVEXX8CpqnqHGAtXd/mU0ZErgAOqeqWvprFeS5l2yvJmtK+raJcoKpnAZXAV0RkYcz8lG0vC4vuaoHobwllwP4M1dJJVRsjuxFUdRXgF5Fx6XhtEfHjfCA/oap/jNMkI9ssUV2Z3GbuazYALwEVMbM6t5eI+IBRpHEXZG91qWqdqra7k78Azk5DORcAV4rIR8DTwKUi8tuYNuneXglrytC2irz2fvf3IeBZYH5Mk5T9PVpYdLcS+B/uGQULgKOqeiDTRYnIxMh+WhGZj/PvVpeG1xXgV8DbqvrjXpqlfZslU1cmtpmInCAio93H+cDlwDsxzVYCN7mPvwC8oO6RyUzWFbNf+0qc40Appar3qGqZqk7DOXj9gqreGNMsrdsrmZoysa3c1y0UkeLIY2AJEHsGZcr+Hn2DsZKhQkSewjlLZpyI1AL34hzsQ1X/FViFczbBbqAFuCVL6voC8DciEgRagetS/QHjugD4EvCmu78b4FvA1KjaMrHNkqkrE9tsEvC4iHhxwul3qvq8iHwP2KyqK3FC7jcishvnG/J1Ka4p2bruEpErgaBb181pqCuuLNheiWrK1LaaADzrfgfyAU+qarWI/DWk/u/RruA2xhiTkO2GMsYYk5CFhTHGmIQsLIwxxiRkYWGMMSYhCwtjjDEJWVgYY4xJyMLCmDRzbzM9oKvJ3dtjTx6MdRnTHxYWxgwtNwOTEzUyZrBZWJgRS0Smicg7IvJLEdkhIk+IyOUiskFE3hOR+e7Pf4kzEM5/icip7rL/W0QedR+f4S5f0MvrjBWRGncdDxN1szcRuVGcgYm2icjD7lXWiMgxEfknEdkqIuvcW3Z8AZgHPOG2z3dX87/cdm+KyMxUbjMzcllYmJHuZOAnwBxgJnADcCHwNZxbiLwDLFTVzwLfAR5wl1sOnCwiVwOPAX+lqi29vMa9wJ/ddazEvS2JiMwCvohzJ9G5QAj4S3eZQmCre4fRl4F7VfUPwGbgL1V1rqq2um0/dds95NZtzKAbUfeGMiaOD1X1TQAReQtYp6oqIm8C03Ducvq4iMzAudVz5J5dYRG5GXgDeFhVN/TxGguBv3CX+08RqXefvwznjqWb3Pv95OOMNwHOWArPuI9/C8S7429EZN6WyOsYM9gsLMxI1x71OBw1Hcb5+/g/wIuqerU4Ay29FNV+BnCM5I4hxLsJmwCPq+o9A1w+IlJzCPubNiliu6GM6dsoYJ/7+ObIkyIyCmf31UJgrHs8oTfrcXcviUglUOo+vw74goiMd+eNEZET3XkenDvngrNr7M/u4yaccceNSSsLC2P69n+BH4jIBsAb9fw/A/+iqu8CtwIPRj7047gPWCgiW3HGINgDoKo7gW/jDJP5BrAG53biAM3A6SKyBbgU+J77/K+Bf405wG1Mytktyo3JQiJyTFWLMl2HMRHWszDGGJOQ9SyMGSQicgvwtzFPb1DVr2SiHmMGk4WFMcaYhGw3lDHGmIQsLIwxxiRkYWGMMSYhCwtjjDEJWVgYY4xJ6P8B6L5ufSOzKFwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1281ceb9438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Best: %f using %s\" % (GSearch_2.best_score_, GSearch_2.best_params_))\n",
    "test_means = GSearch_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = GSearch_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = GSearch_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = GSearch_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(GSearch_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'ｍLog Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_2.png' )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
